package com.ruoyi.web.scheduled;

import cn.hutool.core.util.ObjectUtil;
import com.ruoyi.web.domain.PolicyCrawlArticle;
import com.ruoyi.web.service.PolicyCrawlArticleService;
import com.ruoyi.web.util.HtmlParseUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;

@Slf4j
@Component
public class Crawler {

    @Autowired
    private PolicyCrawlArticleService policyCrawlArticleService;

//    @Scheduled(cron = "0 0/1 * * * ?") // 每分钟执行一次
    @Scheduled(cron = "0 0 0 * * ?") // 每天凌晨0点定时执行
    public void executeTasks() {
        // 山东省工业和信息化厅
        crawlAndSavePolicyCrawlArticle();
    }

    public void crawlAndSavePolicyCrawlArticle() {
        log.info("开始抓取山东工业信息厅 - 政策文件文章：");
        // http://gxt.shandong.gov.cn/col/col103862/index.html 规范性文件
        List<PolicyCrawlArticle> list1 = HtmlParseUtil.getPolicyCrawlArticlePolicyPapersList1();
        list1 = deduplicateByArticleTitleProvince(list1);
        List<PolicyCrawlArticle> list = new ArrayList<>(list1);

        // http://gxt.shandong.gov.cn/col/col103863/index.html 其他文件
        List<PolicyCrawlArticle> list2 = HtmlParseUtil.getPolicyCrawlArticlePolicyPapersList();
        list2 = deduplicateByArticleTitleProvince(list2);
        list.addAll(list2);

        // http://gxt.shandong.gov.cn/col/col103864/index.html 规范性文件清理
        List<PolicyCrawlArticle> list3 = HtmlParseUtil.getPolicyCrawlArticlePolicyPapersList2();
        list3 = deduplicateByArticleTitleProvince(list3);
        list.addAll(list3);

        // http://gxt.shandong.gov.cn/col/col103865/index.html 政策解读
        List<PolicyCrawlArticle> list4 = HtmlParseUtil.getPolicyCrawlArticlePolicyPapersList3();
        list4 = deduplicateByArticleTitleProvince(list4);
        list.addAll(list4);

        if (ObjectUtil.isNotEmpty(list)) {
            policyCrawlArticleService.savePolicyCrawlArticleListBatch(list);
            log.info("{}时 ====政策文件文章保存====", LocalDateTime.now());
        }
    }

    /**
     * 根据文章标题去重
     */
    private List<PolicyCrawlArticle> deduplicateByArticleTitleProvince(List<PolicyCrawlArticle> list) {
        return new ArrayList<>(list.stream()
                .collect(Collectors.toMap(
                        PolicyCrawlArticle::getArticleTitle,
                        Function.identity(),
                        (existing, newItem) -> existing
                ))
                .values());
    }
}